package com.crawler

import com.crawler.KVideo.Companion.EXECUTOR
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import org.openqa.selenium.Keys
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.chrome.ChromeOptions
import org.openqa.selenium.interactions.Actions
import java.io.File
import java.nio.file.Files
import java.nio.file.Paths
import java.util.*
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.Consumer
import kotlin.system.exitProcess

class KVideo {

    companion object {

        /**
         * 小视频首页，按需修改
         */
        private val MAIN_PAGE_URL = "https://www.ixigua.com/home/3276166340814919/hotsoon/"

        /**
         * 存放目录，按需修改
         */
        private val FILE_SAVE_DIR = "D:/video/xigua/"

        /**
         * 线程池，按需修改并行数量。实际开发请自定义避免OOM
         */
        val EXECUTOR = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())

        /**
         * 谷歌浏览器参数
         */
        private val CHROME_OPTIONS = ChromeOptions()
    }


    init {

        // 驱动位置
        System.setProperty("webdriver.chrome.driver", getAbsolutePath() + "\\src\\main\\resources\\chromedriver.exe")
        // 避免被浏览器检测识别
        CHROME_OPTIONS.setExperimentalOption("excludeSwitches", Collections.singletonList("enable-automation"))
    }

    /**
     * 绝对路径
     *
     * @return
     * @throws Exception
     */
    private fun getAbsolutePath(): String {
        return try {
            val directory = File("") //参数为空
            directory.canonicalPath //
        } catch (ex: Exception) {
            throw RuntimeException("")
        }
    }


    /**
     * 获取首页内容
     * @return 首页内容
     * @throws InterruptedException 睡眠中断异常
     */
    @Throws(InterruptedException::class)
    public fun getMainPageSource(): String? {
        val driver = ChromeDriver(CHROME_OPTIONS)
        return try {
            driver[MAIN_PAGE_URL]
            val waitTime = java.lang.Double.valueOf(Math.max(3.0, Math.random() * 5) * 1000).toLong()
            TimeUnit.MILLISECONDS.sleep(waitTime)
            var timeout: Long = 30000
            // 循环下拉，直到全部加载完成或者超时
            do {
                Actions(driver).sendKeys(Keys.END).perform()
                TimeUnit.MILLISECONDS.sleep(waitTime)
                timeout -= waitTime
            } while (!driver.pageSource.contains("已经到底部，没有新的内容啦")
                && timeout > 0
            )
            driver.pageSource
        } finally {
            driver.close()
            driver.quit()
        }
    }

    /**
     * 处理每个小视频
     *
     * @param div 小视频div标签元素
     * @throws Exception 各种异常
     */
    @Throws(Exception::class)
    fun handleItem(div: Element) {
        val href = div.getElementsByTag("a").first().attr("href")
        val src = getVideoUrl("https://www.ixigua.com$href")
        // 有些blob开头的（可能还有其它）暂不处理
        if (src.startsWith("//")) {
            val response = Jsoup.connect("https:$src")
                .timeout(1000 * 5) // 解决org.jsoup.UnsupportedMimeTypeException: Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml. Mimetype=video/mp4, URL=
                .ignoreContentType(true) // The default maximum is 1MB.
                .maxBodySize(100 * 1024 * 1024)
                .execute()
            Files.write(Paths.get(FILE_SAVE_DIR, href.substring(1) + ".mp4"), response.bodyAsBytes())
        } else {
            println("无法解析的src：[$src]")
        }
    }

    /**
     * 获取小视频实际链接
     *
     * @param itemUrl 小视频详情页
     * @return 小视频实际链接
     * @throws InterruptedException 睡眠中断异常
     */
    @Throws(InterruptedException::class)
    private fun getVideoUrl(itemUrl: String): String {
        val driver = ChromeDriver(CHROME_OPTIONS)
        return try {
            //driver.get(itemUrl);
            driver.navigate().to(itemUrl)
            val waitTime = java.lang.Double.valueOf(Math.max(5.0, Math.random() * 10) * 1000).toLong()
            var timeout: Long = 50000
            var v: Element
            /**
             * 循环等待，直到链接出来
             * ※这里可以考虑浏览器驱动自带的显式等待()和隐士等待
             */
            do {
                TimeUnit.MILLISECONDS.sleep(waitTime)
                timeout -= waitTime
            } while ((Objects.isNull(Jsoup.parse(driver.pageSource).getElementById("player_default").also {
                    v = it
                })
                        || Objects.isNull(v.getElementsByTag("video").first().also { v = it }))
                && timeout > 0
            )
            v.attr("src")
        } finally {
            driver.close()
        }
    }
}

fun main() {

    // 获取小视频列表的div元素，批量处理
    val mainDoc = Jsoup.parse(KVideo().getMainPageSource())
    val divItems = mainDoc.select("div[class=\"VerticalFeedCard\"]")
    // 这里使用CountDownLatch关闭线程池，只是避免执行完一直没退出
    val countDownLatch = CountDownLatch(divItems.size)
    divItems.forEach(
        Consumer { item: Element? ->
            EXECUTOR.execute {
                try {
                    item?.let { KVideo().handleItem(it) }
                } catch (e: Exception) {
                    e.printStackTrace()
                }
                countDownLatch.countDown()
            }
        }
    )
    countDownLatch.await()
    EXECUTOR.shutdown()
    exitProcess(0)
}